/*
 * Copyright 2012 Subhabrata Ghosh <subho.ghosh at langene.net>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */



package net.langene.nlp.pg;

//~--- non-JDK imports --------------------------------------------------------

import net.langene.common.IOUtils;
import net.langene.nlp.Context;

import org.apache.commons.configuration.ConfigurationException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

//~--- JDK imports ------------------------------------------------------------

import java.io.File;

import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;

//~--- classes ----------------------------------------------------------------

/**
 *
 * @author Subhabrata Ghosh <subho.ghosh at langene.net>
 */
public class PGIndexer {
    private static final Logger	_logger = LoggerFactory.getLogger(PGIndexer.class);

    //~--- fields -------------------------------------------------------------

    private String			catalogdir  = null;
    private String			catalogfile = null;
    private HashMap<String, PGItem>	items       = new HashMap<String, PGItem>();
    private List<String>		subrex      = new LinkedList<String>();

    //~--- methods ------------------------------------------------------------

    /**
     * Setup the environment.
     * All initialization tasks are performed here.
     * 
     * @throws Exception
     */
    public void setup() throws Exception {
        catalogfile = Context.get().getConfig().getString("net.langene.pg.catalog");

        if ((catalogfile == null) || catalogfile.isEmpty()) {
            throw new ConfigurationException(
                "Invalid COnfiguration : Missing or empty parameter [net.langene.pg.catalog]");
        }

        subrex = Context.get().getConfig().getList("net.langene.pg.indexer.subjects");

        if ((subrex == null) || subrex.isEmpty()) {
            throw new ConfigurationException(
                "Invalid COnfiguration : Missing or empty parameter [net.langene.pg.indexer.subjects]");
        }
    }

    /**
     * Process the catalog files and load the index records into the DB.
     * 
     * @throws Exception
     */
    public void process() throws Exception {
        _logger.info("Processing catalog [" + catalogfile + "]...");

        File	fi = new File(catalogfile);

        catalogdir  = fi.getParent();
        catalogfile = getUncompressedFile(catalogfile);

        DocumentBuilderFactory	domFactory = DocumentBuilderFactory.newInstance();

        domFactory.setNamespaceAware(false);

        DocumentBuilder	builder = domFactory.newDocumentBuilder();
        Document	doc     = builder.parse(catalogfile);
        XPath		xpath   = XPathFactory.newInstance().newXPath();
        XPathExpression	expr    = xpath.compile("/RDF/etext");
        Object		result  = expr.evaluate(doc, XPathConstants.NODESET);
        NodeList	nodes   = (NodeList) result;

        _logger.debug("Numer of nodes found : " + nodes.getLength());

        if ((nodes != null) && (nodes.getLength() > 0)) {
            for (int ii = 0; ii < nodes.getLength(); ii++) {
                if (ii > 0) {
                    Element	elm = (Element) nodes.item(ii - 1);

                    elm.getParentNode().removeChild(elm);
                }

                Element	ne = (Element) nodes.item(ii);
                String	id = ne.getAttribute("rdf:ID");

                if (ii % 1000 == 0) {
                    System.gc();

                    long	tmem = Runtime.getRuntime().totalMemory() / (1024 * 1024);
                    long	umem = tmem - Runtime.getRuntime().freeMemory() / (1024 * 1024);

                    _logger.debug("Memory : " + umem + "/" + tmem + "MB");
                    _logger.debug("Processed [" + ii + "] records from catalog...");
                }

                expr = xpath.compile("./language/ISO639-2/value");

                Element	elang = (Element) expr.evaluate(ne, XPathConstants.NODE);

                if (elang != null) {
                    String	lang = elang.getTextContent();

                    if (lang.compareToIgnoreCase(Context.get().getLanguage()) != 0) {
                        continue;
                    }
                } else {
                    continue;
                }

                boolean	usenode = false;

                expr = xpath.compile("./subject/Bag/li/LCSH/value");

                NodeList	nlsub = (NodeList) expr.evaluate(ne, XPathConstants.NODESET);

                if ((nlsub != null) && (nlsub.getLength() > 0)) {
                    for (int jj = 0; jj < nlsub.getLength(); jj++) {
                        String	subject = nlsub.item(jj).getTextContent();

                        if ((subject != null) &&!subject.isEmpty()) {
                            for (String rex : subrex) {
                                if (rex.startsWith("!")) {
                                    String	rx = rex.replaceFirst("!", "");

                                    if (subject.indexOf(rx) >= 0) {
                                        usenode = false;

                                        break;
                                    }
                                } else if (subject.indexOf(rex) >= 0) {
                                    usenode = true;
                                }
                            }
                        }

                        if (usenode) {
                            break;
                        }
                    }
                }

                if (!usenode) {
                    expr = xpath.compile("./subject/LCSH/value");

                    Element	esub = (Element) expr.evaluate(ne, XPathConstants.NODE);

                    if (esub != null) {
                        String	subject = esub.getTextContent();

                        if ((subject != null) &&!subject.isEmpty()) {
                            for (String rex : subrex) {
                                if (subject.indexOf(rex) >= 0) {
                                    usenode = true;

                                    break;
                                }
                            }
                        }
                    }
                }

                if (usenode) {
                    if ((id != null) &&!id.isEmpty()) {
                        expr = xpath.compile("./friendlytitle");

                        Element	etitle = (Element) expr.evaluate(ne, XPathConstants.NODE);

                        if (etitle != null) {
                            String	title = etitle.getTextContent();
                            PGItem	item  = new PGItem();

                            item.Id    = id;
                            item.Title = title;
                            items.put(item.Id, item);

                            // _logger.debug("Found node [ ID :" + item.Id + "]");
                        }
                    }
                }
            }
        }

        _logger.debug("Finished processing catalog...");

        if (items.size() > 0) {
            loadfiledata();
        }
    }

    private void loadfiledata() throws Exception {
        String	fileprefix =
            Context.get().getConfig().getString("net.langene.pg.indexer.part.prefix");

        if ((fileprefix == null) || fileprefix.isEmpty()) {
            throw new ConfigurationException(
                "Invalid Configuration : Missing or empty parameter [net.langene.pg.indexer.part.prefix]");
        }

        int			count      = 1;
        DocumentBuilderFactory	domFactory = DocumentBuilderFactory.newInstance();

        domFactory.setNamespaceAware(false);

        DocumentBuilder	builder = domFactory.newDocumentBuilder();

        while (true) {
            String	filename = catalogdir + "/" + fileprefix + "-" + count;

            filename = getUncompressedFile(filename);

            if (filename == null) {
                break;
            }

            _logger.debug("Loading from " + filename + "...");

            Document		doc    = builder.parse(filename);
            XPath		xpath  = XPathFactory.newInstance().newXPath();
            XPathExpression	expr   = xpath.compile("/RDF/file");
            NodeList		result = (NodeList) expr.evaluate(doc, XPathConstants.NODESET);

            if ((result != null) && (result.getLength() > 0)) {
                for (int ii = 0; ii < result.getLength(); ii++) {
                    if (ii > 0) {
                        Element	elm = (Element) result.item(ii - 1);

                        elm.getParentNode().removeChild(elm);
                    }

                    if (ii % 1000 == 0) {
                        System.gc();

                        long	tmem = Runtime.getRuntime().totalMemory() / (1024 * 1024);
                        long	umem = tmem - Runtime.getRuntime().freeMemory() / (1024 * 1024);

                        _logger.debug("Memory : " + umem + "/" + tmem + "MB");
                    }

                    Element	elm    = (Element) result.item(ii);
                    String	urlink = elm.getAttribute("rdf:about");

                    if ((urlink != null) &&!urlink.isEmpty()) {
                        if (urlink.endsWith("htm") || urlink.endsWith("html")) {
                            expr = xpath.compile("./isFormatOf");

                            Element	idelem = (Element) expr.evaluate(elm, XPathConstants.NODE);

                            if (idelem != null) {
                                String	id = idelem.getAttribute("rdf:resource");

                                if ((id != null) &&!id.isEmpty()) {
                                    if (id.startsWith("#")) {
                                        id = id.replaceFirst("#", "");
                                    }

                                    if (items.containsKey(id)) {
                                        PGItem	item = items.get(id);

                                        item.Url = urlink;
                                        _logger.debug("Saving URL [" + item.Url + "]...");
                                        PGDBHelper.save(item);
                                    }
                                }
                            }
                        }
                    }
                }
            }

            count++;
        }
    }

    //~--- get methods --------------------------------------------------------

    private String getUncompressedFile(String filename) throws Exception {
        File	fi    = new File(filename + ".rdf");
        String	fname = fi.getName();

        if (fi.exists()) {
            return fi.getAbsolutePath();
        }

        fi = new File(filename + ".zip");

        if (!fi.exists()) {
            return null;
        }

        String	tempdir = IOUtils.uncompressedFile(fi.getAbsolutePath());

        return IOUtils.findfirst(tempdir, fname);
    }
}


//~ Formatted by Jindent --- http://www.jindent.com
